package com.alex.keywordtool;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Collection;
import java.util.LinkedHashSet;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;

public class ParseTopKeywords {

	public static void main(String[] args) throws Exception {
		Collection<String> keywords = new LinkedHashSet<String>();
		for (File file : FileUtils.listFiles(new File("/home/alex/Desktop/adwords_adult/"), null, false)) {
			BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-16"));
			String line = null;
			while ((line = reader.readLine()) != null) {
				String[] parts = line.split("\t");
				try {
					float competition = Float.parseFloat(parts[1]);
					if(competition > 0.1) {
						keywords.add(parts[0]);
					}
				} catch (Exception e) {
				}
			}
			reader.close();
		}
		
		System.out.println(keywords.size());
		FileUtils.writeStringToFile(new File("/home/alex/Desktop/adult_top.txt"), StringUtils.join(keywords, "\n"));
	}

}
